/**
 * 明星绯闻
 * http://www.jiaodong.net/ent/mingxing/
 */
var cheerio = require("cheerio");//引用抓取html结构的node模块
var mkdirp  = require("mkdirp");  //引用创建文件夹的node模块
var iconv   = require("iconv-lite"); //引用字符集node模块
var commen  = require("../allocation/commen");//引用公共操作的自定义文件
var data    = require("../allocation/sql");  //引用数据库操作的自定义文件
var async   = require("async");  //引用同步执行

var url       = "http://www.jiaodong.net/ent/mingxing/";
var chanel    = "mingxingfeiwen";
var newsType  = 9;
var imgSite   = "";
var website   = "";
var newsLists = [];   //存放新闻列表
var logs      = [];   //存放日志信息


//创建存放图片的文件夹及路径(20160901)
var imgPath               = commen.createDownFile().imgPath;

mkdirp.sync(imgPath);
logs.push({
    start_time:new Date().toLocaleString(),
    article_type_id:newsType,
    thread:chanel
});

//定义获取新闻列表
function getNewsList(url,callback)
{
    commen.load(url,function(err,body){
        if(err)
        {
            logs[0].location="列表请求";
            logs[0].message="错误："+err;
            logs[0].level=0;
            logs[0].end_time=new Date().toLocaleString();
            data.insertLog(logs[0],function(err,result){
                if(err) throw err;
                if(result>0)
                {
                    console.log("列表请求出错了！！！!");
                    process.exit(0);
                }
            });
            return err;
        }
        var  $  =cheerio.load(body.toString());
        var articleList = [];
        //获取新闻列表
        $(".litxt li").each(function (index) {
            if (index==21) {
                return false; 
            }
            if ($(this).find("a").attr("href")!=undefined) {
                articleList.push(website+$(this).find("a").attr("href"));
            }
        });
        // console.log(articleList);
        callback(null, articleList);
        
        // process.exit(0);
    });
}

//定义获取新闻内容
function getNewsContent(url,callback)
{
    commen.load(url,function(err,body){
        if(err)
        {
            logs[0].location="内容请求";
            logs[0].message="错误："+err;
            logs[0].level=0;
            logs[0].end_time=new Date().toLocaleString();
            data.insertLog(logs[0],function(err,result){
                if(err) throw err;
                if(result>0)
                {
                    console.log("内容请求出错了！！！");
                    process.exit(0);
                }
            });
            return err;
        }
        var html        = iconv.decode(body, 'gb2312');
        var $           = cheerio.load(html.toString(), {decodeEntities: false});
        var news        = [];
        var picUrls     = []; //存放图片的数组
        var allContent  = "";  //存放排版好的ubb格式的内容
        
        //imgSite = url.substring(0,url.lastIndexOf("/")+1);
        if($("#newcontent img").length>0)
        {
            for(var i=0;i<$("#newcontent img").length;i++)
            {
                if($($("#newcontent img")[i]).attr("src").indexOf("title")>0)
                {
                    $($("#newcontent img")[i]).remove();
                    continue;
                }
                //改
                if($($("#newcontent").find("img")[i]).closest("center"))
                {
                    $($("#newcontent").find("img")[i]).closest("center").replaceWith("<p>"+$($("#newcontent").find("img")[i]).parent().html()+"</p>");
                }
                picUrls.push(imgSite+$($("#newcontent img")[i]).attr("src"));
                // $($("#newcontent img")[i]).html("[text-align:center]"+imgSite+$($("#newcontent img")[i]).attr("src")+"[/img]");
                // $($("#newcontent img")[i]).removeAttr("title");
                // $($("#newcontent img")[i]).removeAttr("alt");
                // $($("#newcontent img")[i]).removeAttr("src");
            }
        }
        $("#newcontent script").remove();
        $("#newcontent style").remove();
        $("#newcontent #news_more_page_div_id").remove();
        //去除内容html中的空格和样式
        if($("#newcontent").html()==null)
        {
            logs[0].location="解析标签";
            logs[0].message="去除标签错误：Cannot read property 'replace' of null";
            logs[0].level=0;
            logs[0].end_time=new Date().toLocaleString();
            data.insertLog(logs[0],function(err,result){
                if(err) throw err;
                if(result>0)
                {
                    console.log("去除标签出错了！！！");
                    process.exit(0);
                }
            });
            return err;
        }
        else
        {
            $("#newcontent").html($("#newcontent").html().replace(/<!--[\w\W\r\n]*?-->/gmi,'').trim());
            // $("#newcontent").html($("#newcontent").html().replace(/<([a-zA-Z1-6]+)(\s*[^>]*)?>/g, "<$1>"));
        }
         for (var i = 0; i <$("#newcontent").find("p").length; i++) 
             {

                    if($($("#newcontent").find("p")[i]).find("img")=="")
                    {
                        if($($("#newcontent").find("p")[i]).text().trim()=="")
                       {
                          $($("#newcontent").find("p")[i]).remove();
                       }
                    }
             }
         for (var i = 0; i <$("#newcontent").find("p").length; i++) {

                if($($("#newcontent").find("p")[i]).find("img")=="")
                {

                         if($($("#newcontent").find("p")[i]).attr("align")=="center")
                         {

                          //说明属性align:center使得文字居中
                          $($("#newcontent").find("p")[i]).text("[text-align:center]"+$($("#newcontent").find("p")[i]).html());
                      
                         }
                         else
                         {

                             //说明无align属性，判断是否有style样式
                             if($($("#newcontent").find("p")[i]).attr("style")==undefined)
                             {
                                 // 说明无style
                                $($("#newcontent").find("p")[i]).text("[&emsp][&emsp]" + $($("#newcontent").find("p")[i]).html());
            
                             }
                             else
                             {
                                if($($("#newcontent").find("p")[i]).attr("style").indexOf("center")>0)
                                {
                                 //说明居中
                                  $($("#newcontent").find("p")[i]).text("[text-align:center]"+$($("#newcontent").find("p")[i]).html());
                      
                                }
                                else
                                {
                            
                                   //说明不居中
                                   $($("#newcontent").find("p")[i]).text("[&emsp][&emsp]" + $($("#newcontent").find("p")[i]).html());
                

                                }

                             }

                         }

                }
                else
                {
                    $($("#newcontent").find("p")[i]).find("img").parent("p").html("[text-align:center][img]"+imgSite+$($("#newcontent").find("p")[i]).find("img").attr("src")+"[/img]");
                
                }
                

            }
        //找到内容中所有的p标签
        for (var i = 0; i <$("#newcontent p").length; i++) {
            if($($("#newcontent p")[i]).find("img")=="")
            {
                $($("#newcontent p")[i]).text( $($("#newcontent p")[i]).html());
            }
        }
         $("#newcontent").html($("#newcontent").html().replace(/<([a-zA-Z1-6]+)(\s*[^>]*)?>/g, "<$1>"));
        if($("#newcontent").html()==null)
        {
            logs[0].location="解析标签";
            logs[0].message="转换标签错误：Cannot read property 'replace' of null";
            logs[0].level=0;
            logs[0].end_time=new Date().toLocaleString();
            data.insertLog(logs[0],function(err,result){
                if(err) throw err;
                if(result>0)
                {
                    console.log("转换标签出错了！！！");
                    process.exit(0);
                }
            });
            return err;
        }
        else
        {
            //替换内容中所有的&nbsp;$gt等标签为ubb格式  .replace(/(<\/?span.*?>)/g,"")
            allContent = $("#newcontent").html().trim().replace(/\s/g, "").replace(/>/g, "]").replace(/</g, "[").replace(/&rdquo;/g, "[&rdquo]").replace(/&ldquo;/g,"[&ldquo]").replace(/&nbsp;/g,"[&nbsp]").replace(/&amp;/g,"[&amp]").replace(/&gt;/g,"[&gt]").replace(/&lt;/g,"[&lt]");
        }  
           // console.log(allContent);
        //将插入数据库的值存入数组news中
        news.push({
            title:$(".title").text().trim(),
            source:"胶东在线",
            author:$(".pt20").text().trim(),
            date:new Date().toLocaleString(),
            typeID:newsType,
            chanel:chanel,
            content:allContent,
            picUrls:picUrls
        });
        callback(null, news[0]);
        // console.log(news[0]);
    });
}

//调用方法，依次获取列表和内容，并存入数据库
async.series([
    // 获取新闻列表
    function (done) {
        getNewsList(url,function(err,list){
            newsLists = list;
            done(err);
        });
    },
    function (done) {
        async.eachSeries(newsLists, function (item, next) {
            getNewsContent(item, function (err,detail) {
                if (err) console.error(err.stack);
                data.insertData(detail,imgPath,function(err,result){
                   // console.log(detail);  //数组(title,source等，打印内容，就是detail.content)
                    if(err)
                    {
                        logs[0].location="入库";
                        logs[0].message="错误："+err;
                        logs[0].level=0;
                        logs[0].end_time=new Date().toLocaleString();
                        data.insertLog(logs[0],function(err,result){
                            if(err) throw err;
                            if(result>0)
                            {
                                console.log("入库出错了！！！");
                                process.exit(0);
                            }
                        });
                        return err;
                    }
                    else if(result>0)
                     {
                         console.log("采集成功");
                     }
                });
                next();
            });
        }, done);
    }
],function (err) {
    if (err) console.error(err.stack);

    logs[0].location="无";
    logs[0].message="正确";
    logs[0].level=1;
    logs[0].end_time=new Date().toLocaleString();
    data.insertLog(logs[0],function(err,result){
        if(err) throw err;
        if(result>0)
        {
            console.log("写入日志成功!");
            process.exit(0);
        }
    });
});
